package com.shuai.AI.ai_app.rag.other;

import java.util.List;
import org.springframework.ai.document.Document;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.stereotype.Component;

/** 自定义基于 Token 的切词器 */
@Component
class MyTokenTextSplitter {
  public List<Document> splitDocuments(List<Document> documents) {
    TokenTextSplitter splitter = new TokenTextSplitter();
    return splitter.apply(documents);
  }

  public List<Document> splitCustomized(List<Document> documents) {
    TokenTextSplitter splitter = new TokenTextSplitter(200, 100, 10, 5000, true);
    return splitter.apply(documents);
  }
}
